# encoding:utf-8
"""
@project:Get_DL_Dataset
@author:liuzeyu
@time:2022-07-29
@version:v1.1
@description:支持将数据集导出为feather格式，以提升读写速率并减少空间占用
"""
import configparser
import configparser as cp
import os
import time
import pandas as pd


def time_dec(func):
    def wrapper(*args, **kwargs):
        """
        计时器装饰器
        :param args:
        :param kwargs:
        :return:
        """
        t = time.perf_counter()
        res = func(*args, **kwargs)
        print(func.__name__, '操作执行用时：', time.perf_counter() - t)
        return res

    return wrapper


def tradedt_pegging(date, df):
    """
    根据时间反查位置
    :param date:
    :param df:
    :return:
    """
    obj = df.iloc[(df['trade_dt'] - date).abs().argsort()[0], :].tolist()
    t = obj[1]
    return t


def stockcode_pegging(stock, df):
    """
    根据股票反查位置
    :param stock:
    :param df:
    :return:
    """
    obj = df.loc[(df['stockcode'] == stock), :]
    s = obj.iloc[0, 1]
    return s


def sample_clearner(select):
    """
    样本清洗器，判断取出的样本是否满足要求（任意一行不含空值）
    :param select:
    :return:
    """
    bool = 1
    judge = select.isnull().any()
    for i in judge.tolist():
        if i == False:
            continue
        else:
            bool = 0
            break
    return bool


def sample_concat(samples, select, bool):
    """
    样本拼接器，满足要求的样本保留，不满足的样本丢弃
    :param samples:
    :param select:
    :param bool:
    :return:
    """
    if bool == 1:
        # samples = samples.append(select, ignore_index=True)
        samples = pd.concat([samples, select], ignore_index=True)
    else:
        pass
        # print('此样本不满足无空值条件，该样本已被舍弃')
    return samples


def cutter(start_index, i, Samples, Data: pd.DataFrame):
    """
    样本切割、有效性判断、拼接
    :param start_index:
    :param i:
    :param Samples:
    :param Data:
    :return:
    """
    s = Data.iloc[start_index + i:start_index + 30 + i, :]
    bool = sample_clearner(select=s)
    Samples = sample_concat(samples=Samples, select=s, bool=bool)
    return Samples


class DL_Dataset:
    def __init__(self, config: configparser.ConfigParser):
        self.config = config
        self.starttime = config.getint('time', 'starttime')
        self.endtime = config.getint('time', 'endtime')
        self.S1_inputpath = config.get('Path', 'S1_inputpath')
        self.S1_outputpath = config.get('Path', 'S1_outputpath')
        self.S2_outputpath = config.get('Path', 's2_outputpath')
        self.stock_num = config.getint('para', 'stock_num')
        self.stockcode_table = pd.read_csv(config.get('Path', 'stockcode_table_path'))
        self.trade_dt_table = pd.read_csv(config.get('Path', 'trade_dt_table_path'))
        self.stocklist = pd.read_csv(config.get('Path', 'stocklist'))['stockcode'].tolist()[0:config.getint('para', 'stock_num')]

    @time_dec
    def sourcedata(self):
        Data = pd.DataFrame()
        count = 0
        for file in os.listdir(self.S1_inputpath):
            file_name = self.S1_inputpath + file
            with open(file=file_name):
                df = pd.read_parquet(file_name)
            if count == 0:
                Data = df
                count = count + 1
                print('第', count, '个因子已载入')
            else:
                Data = pd.concat([Data, df.iloc[:, 2]], axis=1)
                count = count + 1
                print('第', count, '个因子已载入')
        return Data

    @time_dec
    def get_dataset(self, sourcedata, storage_format: str = 'feather') -> pd.DataFrame:
        # Samples = pd.DataFrame()
        Dataset = pd.DataFrame()
        start_index = stockcode_pegging(stock='000001.SZ', df=self.stockcode_table) + tradedt_pegging(date=self.starttime, df=self.trade_dt_table)
        end_index = stockcode_pegging(stock='000001.SZ', df=self.stockcode_table) + tradedt_pegging(date=self.endtime, df=self.trade_dt_table)
        samplenum = end_index - start_index
        cyclenum = samplenum - 30 - 1
        for i in range(cyclenum):
            Samples = pd.DataFrame()
            for s in self.stocklist:
                start_index = stockcode_pegging(stock=s, df=self.stockcode_table) + tradedt_pegging(self.starttime, self.trade_dt_table)
                end_index = stockcode_pegging(stock=s, df=self.stockcode_table) + tradedt_pegging(self.endtime, self.trade_dt_table)
                Samples = cutter(start_index=start_index, i=i, Samples=Samples, Data=sourcedata)
            # Dataset = Dataset.append(Samples, ignore_index=True)
            Dataset = pd.concat([Dataset, Samples], ignore_index=True)
            print(i, '次循环结束，共', cyclenum, '次。目前进度为：', '{:.2%}'.format(i / cyclenum))
        print('Dataset已制成，在给定的时间范围(', self.starttime, '-', self.endtime, ')内总计构造了', len(Dataset), '条数据,剔除了',
              len(Dataset) - (len(self.stocklist) * cyclenum), '条无效数据')
        if storage_format == "csv":
            # 输出为CSV文件
            output_file_name = self.S1_outputpath + 'Dataset.csv'
            Dataset.to_csv(output_file_name, index=False)
        elif storage_format == "feather":
            # 输出为feather文件
            output_file_name = self.S1_outputpath + 'Dataset.f'
            Dataset.to_feather(output_file_name)
        else:
            print("仅支持csv或feather格式输出")
        print('Dataset已导出到S1_dataset文件夹下')
        return Dataset

    @time_dec
    def get_labels(self, Dataset: pd.DataFrame, storage_format: str = "feather") -> pd.DataFrame:
        print('正在抽取样本label...')
        labels = pd.DataFrame()
        for i in range(int(len(Dataset) / 30)):
            index = (i + 1) * 30
            labels = labels.append(Dataset.iloc[index - 1, :])
        # print('样本label抽取进度：', '{:.2%}'.format(i * 30 / len(Dataset)))
        if storage_format == "csv":
            # 输出为CSV文件
            output_file_name = self.S2_outputpath + 'Label.csv'
            labels.to_csv(path_or_buf=output_file_name, index=False)
        elif storage_format == "feather":
            # 输出为feather文件
            output_file_name = self.S2_outputpath + 'Label.f'
            labels.reset_index().to_feather(output_file_name)
        else:
            print("仅支持csv或feather格式输出")
        print('样本label已抽取，总计抽取了', len(labels), '个样本Label')
        return labels


if __name__ == '__main__':
    # 加载配置文件
    config = cp.ConfigParser()
    config.read('config.ini')

    # 实例化DL_Dataset对象
    # Dest = DL_Dataset(config=config)
    # 加载源数据
    # Data = Dest.sourcedata()
    # 制成Dataset
    # Dataset = Dest.get_dataset(sourcedata=Data, storage_format='feather')
    # 抽取样本label
    # Dest.get_labels(Dataset=Dataset, storage_format='feather')

    # 自动化全流程（加载源数据->制成Dataset->抽取样本label）
    # D = DL_Dataset(config=config)
    # D.get_labels(Dataset=D.get_dataset(sourcedata=D.sourcedata()))

    # 查看csv格式的Dataset
    # Dataset = pd.read_csv(config.get('Path', 'S1_outputpath') + 'Dataset.csv')
    # 查看feather格式的Dataset
    # Dataset = pd.read_feather(config.get('Path', 'S1_outputpath') + 'Dataset.f')
    # print(Dataset)

    # 查看csv格式的Label
    # labels = pd.read_csv(config.get('Path', 'S2_outputpath') + 'Label.csv')
    # 查看feather格式的Label
    # labels = pd.read_feather(config.get('Path', 'S2_outputpath') + 'Label.f')
    # labels = labels.iloc[:, 1:]
    # print(labels)
